package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.MalaModuleTask;
import com.spider.silence.task.XiQinHuiGuanModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

/**
 * @author liyuanguo
 * @version V1.0
 * @time 2017/3/28 9:35
 * @description
 */
public class XiQinHuiGuanDownLoader extends AbstractCommonDownLoader implements Cloneable {
    private Logger logger = Logger.getLogger(XiQinHuiGuanDownLoader.class);
    private String[] modules = {""};
    private Map<String, File> files = null;

    public Map<String, File> getFiles() {

        return files;
    }


    public void generateTask() {
        Document doc;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("XiQinHuiGuan.TimeOut"))) {
            System.out.println("西秦会馆begin....");
            try {
                Page pageContent = null;
                pageContent = getHtml("http://bbs.zg163.net", true);
                if (pageContent.getStatusCode() == 200) {
                    doc = pageContent.getHtml().getDocument();
                    Elements elementsData = doc.select("div[id^='category_']").select("dl");
                    Elements elementss = new Elements();
                    for (int i = 0; i < elementsData.size(); i++) {
                        String updateTime = "";
                        updateTime = elementsData.get(i).select("dd").get(1).select("a").text();
                        if (!"".equals(updateTime) && DateUtils.chargeDateTime(updateTime)) {
                            Elements moduleUrls = elementsData.get(i).select("dt").select("a");
                            if (null != moduleUrls && moduleUrls.size() > 0) {
                                elementss.add(moduleUrls.get(0));
                            }
                        }
                    }
                    // 获取所有的模块
                    for (int i = 0; i < elementss.size(); i++) {
                        XiQinHuiGuanModuleTask task = new XiQinHuiGuanModuleTask(elementss.get(i).attr("href"), elementss.size(), i);
                        CommonTaskManager.add("xiQinHuiGuanModule", task);
                    }
                    break;
                }
            } catch (Exception e1) {
                try {
                    Thread.sleep(5000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                System.err.println("获取任务失败, 重新尝试获取");
                logger.error(e1.getMessage(), e1);
                continue;
            } finally {
                nowTime = System.currentTimeMillis();
            }
        }

    }

    public static void main(String[] args) {
        String url = "http://bbs.zg163.net/thread-4966350-1-1.html";
        XiQinHuiGuanDownLoader d = new XiQinHuiGuanDownLoader();
        d.crawlItem(url);
    }

    public JSONObject crawlItem(String url) {
        files = new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();
        int page = 1;
        //System.out.println("爬取的：" + url);
        try {
            // Document document=get(url,true);
            Document document = null;
            boolean flag = true;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("XiQinHuiGuan.TimeOut"))) {
                try {
                    // doc= get("http://www.mala.cn/forum.php",true);
                    Page pageContent = null;
                    pageContent = getHtml(url, true);
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        try {
                            //System.out.println("爬取言论第一页:" + url + "状态码：" + pageContent.getStatusCode());
                            document = pageContent.getHtml().getDocument();
                            document.select(".aimg_tip").remove();
                            String title = document.title();
                            Elements pageInfos = document.select("#postlist");
                            if (null != pageInfos && pageInfos.size() > 0) {
                                Element pageInfo = pageInfos.get(0);
                                String content = "";
                                if (null != pageInfo.select(".t_f") && pageInfo.select(".t_f").size() > 0 && !"".equals(pageInfo.select(".t_f").get(0).html())) {

                                    // 爬取第一页上帖子的内容
                                    int read_count = Integer.parseInt(pageInfo.select("span.xi1").get(0).text());// 阅读量
                                    // System.out.println("阅读量："+read_count);
                                    int reply_count = Integer.parseInt(pageInfo.select("span.xi1").get(1).text());// 回复量
                                    // System.out.println("回复数："+reply_count);
                                    // page= (int)
                                    // Math.ceil(((double)reply_count+1)/20d);//页数
                                    Element pageI = document.select("#pgt").select(".pgt").get(0);
                                    if (null != pageI.html() && !"".equals(pageI.html())) {
                                        String pageString = pageI.select("span").text();
                                        page = Integer.valueOf(pageString.split(" ")[1]);
                                    }
                                    // System.out.println("页数："+page);
                                    String publish_time = "";// 发布时间
                                    if (null != pageInfo.select(".authi").select("em").get(0).select("span") && pageInfo.select(".authi").select("em").get(0).select("span").size() > 0) {
                                        String timeStrr = pageInfo.select(".authi").select("em").get(0).select("span").attr("title");
                                        publish_time = DateUtils.strToFormatStr(getTime(timeStrr) + ":00");
                                    } else {
                                        publish_time = DateUtils.strToFormatStr(getTime(pageInfo.select(".authi").select("em").text()) + ":00");
                                    }

                                    //System.out.println("发布时间："+publish_time);
                                    content = pageInfo.select(".t_f").get(0).text();// 发布内容
                                    Elements imgEles = pageInfo.select(".t_f").get(0).select("img[zoomfile]");
                                    JSONArray img_urls = new JSONArray();
                                    for (int imgIndex = 0; imgIndex < imgEles.size(); imgIndex++) {
                                        Element imgEle=imgEles.get(imgIndex);
                                        String img_url=imgEle.absUrl("zoomfile");
                                        img_urls.add(img_url);
                                    }
                                    //System.out.println("发布内容：" + content);
                                    String idUrl = pageInfo.select(".authi").select("a").get(0).attr("href");
                                    String[] idUrls = idUrl.split("-");
                                    String publish_user_id = idUrls[2].substring(0, 6);// 发布者的ID
                                    // System.out.println("用户id："+publish_user_id);
                                    String publish_user = pageInfo.select(".authi").select("a").get(0).text();// 发布用户
                                    //System.out.println("用户:"+publish_user);
                                   /* Elements photos =document.select("#postlist").select("div[id^='post_']:gt(0)");
                                    Element p= photos.get(0);
                                    Element ees=p.select(".m").get(0);
                                    String publish_user_photo =ees.select("img").get(0).attr("src");*/
                                    String publish_user_photo = "http://ucserver.higo.cn/avatar.php?uid=" + publish_user_id + "&size=middle";
                                    //System.out.println("用户头像："+publish_user_photo);
                                    String[] urlParam = url.split("-");
                                    String id = urlParam[1];// 帖子的Id
                                    // System.out.println("帖子id："+id);
                                    try {
                                        String like = pageInfo.select("#recommendv_add").text();
                                        // System.out.println("点赞数："+like);
                                        String dislike = pageInfo.select("#recommendv_subtract").text();
                                        // System.out.println("反对数："+dislike);
                                        if (!"".equals(like) && like != null) {
                                            int like_count = Integer.parseInt(like);
                                            data.put("like_count", like_count);
                                        } else {
                                            data.put("like_count", 0);
                                        }
                                        if (!"".equals(dislike) && dislike != null) {
                                            int dislike_count = Integer.parseInt(dislike);
                                            data.put("dislike_count", dislike_count);
                                        } else {
                                            data.put("dislike_count", 0);
                                        }
                                    } catch (Exception e) {
                                        logger.error(e.getMessage(),e);
                                    }

                                    data.put("reply_count", reply_count);
                                    data.put("read_count", read_count);
                                    data.put("title", (title));
                                    data.put("url", url);
                                    data.put("publish_time", publish_time);
                                    Date now = new Date();
                                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                    data.put("spider_time", sdf.format(now));
                                    data.put("reproduce_count", 0);
                                    data.put("content", (content));
                                    data.put("publish_user_id", publish_user_id);
                                    data.put("publish_user", (publish_user));
                                    data.put("publish_user_photo", publish_user_photo);
                                    data.put("img_urls",img_urls);
                                    data.put("id", id);

                                    long tsp = DateUtils.dateToTimestamp(DateUtils.parseDate(publish_time)).getTime();
                                    String filePrefix = "XiQiuHuiGuan";//文件前缀
                                    String fileName = filePrefix + "_" + tsp + "_" + publish_user_id + "_" + id;
                                    String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-", "-");
                                    String speechesId = "XiQiuHuiGuan_speeches_" + publishDate + "_" + fileName;
                                    File file = FileUtils.createTempFile(speechesId + "_pageNo_1", document.html().toString());
                                    files.put(speechesId + "_pageNo_1", file);
                                    //帖子首页的文件

                                    Elements replyEntities = document.select("#postlist").select("div[id^='post_']:gt(2)");
                                    JSONArray reply_nodes = new JSONArray();
                                    // System.out.println("回复："+replyEntities);

                                    // 爬取第一页的回复
                                    for (int i = 0; i < replyEntities.size(); i++) {
                                        try {
                                            Element replyUserInfo = replyEntities.get(i).select("td.pls").get(0);
                                            //System.out.println("回复用户信息："+replyUserInfo);
                                            JSONObject reply = new JSONObject();
                                            String replyUser = "";
                                            String replyUserId = "";
                                            Elements es = replyUserInfo.select(".authi").select("a");
                                            if (null != replyUserInfo.select(".authi").select("a")
                                                    && replyUserInfo.select(".authi").select("a").size() > 0) {
                                                replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                                String idUrl1 = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                                String[] idUrls1 = idUrl1.split("-");
                                                replyUserId = idUrls1[2].substring(0, 6);
                                            } else if (null != replyUserInfo.select("em")
                                                    && replyUserInfo.select("em").size() > 0) {
                                                replyUser = replyUserInfo.select("em").get(0).text();
                                            }
                                            //System.out.println("回复用户名："+replyUser);
                                            // String
                                            // homeUrl=replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                            String homeUrl = "";
                                            // String
                                            // replyUserId=getUrlParams(homeUrl).get("uid").toString();
                                            // System.out.println("头像内容："+replyUserInfo.select(".avatar"));
                                            String replyUserPhoto = "http://ucserver.higo.cn/avatar.php?uid=" + replyUserId + "&size=middle";
                                            //System.out.println("回复者头像："+replyUserPhoto);
                                            String replyContent = replyEntities.get(i).select(".plc").select(".pct").get(0).text();
                                            Elements replyImgEles=replyEntities.get(i).select(".plc").select(".pct").get(0).select("img[zoomfile]");
                                            JSONArray reply_img_urls=new JSONArray();
                                            for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                                Element imgEle=replyImgEles.get(imgIndex);
                                                String img_url=imgEle.absUrl("zoomfile");
                                                reply_img_urls.add(img_url);
                                            }
                                            //System.out.println("回复内容："+replyContent);
                                            String replyTime = "";
                                            if (null != replyEntities.get(i).select(".authi").select("em").get(0).select("span") && replyEntities.get(i).select(".authi").select("em").get(0).select("span").size() > 0) {
                                                replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".authi").select("em").get(0).select("span").attr("title")) + ":00");
                                            } else {
                                                replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".authi").select("em").text()) + ":00");
                                            }
                                            //System.out.println("回复时间："+replyTime);
                                            String replyUrl = url + "#" + replyEntities.get(i).id();
                                            // System.out.println("回复url："+replyUrl);

                                            reply.put("publish_user", replyUser);
                                            reply.put("publish_user_id", replyUserId);
                                            reply.put("publish_user_photo", replyUserPhoto);
                                            reply.put("content", replyContent);
                                            reply.put("publish_time", replyTime);
                                            reply.put("url", replyUrl);
                                            reply.put("parent_id", id);
                                            reply.put("ancestor_id", id);
                                            reply.put("img_urls",reply_img_urls);
                                            reply_nodes.add(reply);
                                        } catch (Exception e) {
                                            logger.error(e.getMessage(),e);
                                            continue;// 因为禁言无法获取的情况
                                        }

                                    }

                                    JSONArray replys = new JSONArray();
                                    List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();

                                    // 爬取2页之后每页的回复
                                    if (page >= 2) {
                                        int threadNum = Integer.valueOf(PropertiesUtil.getVal("XiQinHuiGuan.ReplyThreadCount"));
                                        List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                        // 线程池 初始化十个线程 和JDBC连接池是一个意思 实现重用
                                        ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                        try {
                                            ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                            for (int i = 0; i < splitPages.size(); i++) {
                                                XiQinHuiGuanDownLoader downLoader = this.cloneDownLoader();
                                                XiQinHuiGuanDownLoader.ReplyCrawler callable = new XiQinHuiGuanDownLoader.ReplyCrawler(downLoader, splitPages.get(i), url, id, speechesId);
                                                FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                                futureTasks.add(futureTask);
                                                executorService.submit(futureTask);
                                            }
                                            int count = 0;

                                            for (FutureTask<JSONArray> futureTask : futureTasks) {
                                                JSONArray replyArrays = futureTask.get();
                                                for (int i = 0; i < replyArrays.size(); i++) {
                                                    JSONObject object = replyArrays.getJSONObject(i);
                                                    replys.add(object);
                                                }
                                            }
                                            if (replys.size() > 0 && replys != null) {
                                                for (int i = 0; i < replys.size(); i++) {
                                                    JSONObject reply_object = replys.getJSONObject(i);
                                                    reply_nodes.add(reply_object);
                                                }
                                            }
                                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                                for (int i = 0; i < reply_nodes.size(); i++) {
                                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                    Map<String, Object> map = reply_object;
                                                    list.add(map);
                                                }
                                            }
                                            Collections.sort(list, new Comparator<Map<String, Object>>() {

                                                public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                                    String s1 = o1.get("publish_time").toString();
                                                    String s2 = o2.get("publish_time").toString();
                                                    int flag = s1.compareTo(s2);
                                                    return flag;
                                                }
                                            });
                                        } catch (Exception ee) {
                                            logger.error(ee.getMessage(),ee);
                                            ee.printStackTrace();
                                        } finally {
                                            executorService.shutdown();
                                        }
                                    } else {
                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                Map<String, Object> map = reply_object;
                                                list.add(map);
                                            }
                                        }
                                    }
                                    data.put("reply_nodes", list);
                                } else {

                                }
                            } else {

                            }

                        } catch (Exception e) {

                            logger.error(e.getMessage(), e);
                        } finally {
                            flag = false;
                        }

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                    } else {

                    }

                } catch (Exception e1) {
                    //System.out.println("异常信息：" + e1.getMessage());
                    logger.error(e1.getMessage(), e1);
                } finally {
                    if (flag == true) {

                        item = new JSONObject();
                        data = new JSONObject();
                    }
                    nowTime = System.currentTimeMillis();
                }
            }

        } catch (Exception e) {
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(), e);
        }
        item.put("data", data);
        return item;
    }

    public XiQinHuiGuanDownLoader cloneDownLoader()
            throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        XiQinHuiGuanDownLoader loader = new XiQinHuiGuanDownLoader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public class ReplyCrawler implements Callable<JSONArray> {
        private XiQinHuiGuanDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;

        @Override
        public JSONArray call() throws Exception {
            JSONArray array = new JSONArray();
            for (int k = 0; k < pages.size(); k++) {
                JSONArray arrayPage = new JSONArray();
                try {

                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("XiQinHuiGuan.TimeOut"))) {
                        try {
                            String[] urlParam = url.split("-");
                            // System.out.println("页码："+pages.get(k));
                            String getUrl = urlParam[0] + "-" + urlParam[1] + "-" + pages.get(k) + "-" + urlParam[3];
                            Document document = null;
                            Page pageContent = null;
                            pageContent = downLoader.getHtml(getUrl, true);
                            if (null == pageContent) {

                            } else if (pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                            } else if (pageContent.getStatusCode() == 200) {
                                document = pageContent.getHtml().getDocument();
                                document.select(".aimg_tip").remove();
                                File file = FileUtils.createTempFile(speeches_id + "_pageNo_" + pages.get(k), document.html().toString());
                                files.put(speeches_id + "_pageNo_" + pages.get(k), file);
                                Elements replyEntities = document.select("#postlist").select("div[id^='post_']:gt(1)");
                                JSONArray reply_nodes = new JSONArray();
                                for (int i = 0; i < replyEntities.size(); i++) {
                                    try {
                                        Element replyUserInfo = replyEntities.get(i).select("td.pls").get(0);
                                        // System.out.println("回复用户信息："+replyUserInfo);
                                        JSONObject reply = new JSONObject();
                                        String replyUser = "";
                                        String replyUserId = "";
                                        Elements es = replyUserInfo.select(".authi").select("a");
                                        if (null != replyUserInfo.select(".authi").select("a")
                                                && replyUserInfo.select(".authi").select("a").size() > 0) {
                                            replyUser = replyUserInfo.select(".authi").select("a").get(0).text();
                                            String idUrl1 = replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                            String[] idUrls1 = idUrl1.split("-");
                                            replyUserId = idUrls1[2].substring(0, 6);
                                        } else if (null != replyUserInfo.select("em")
                                                && replyUserInfo.select("em").size() > 0) {
                                            replyUser = replyUserInfo.select("em").get(0).text();
                                        }
                                        //System.out.println("回复用户名："+replyUser);
                                        // String
                                        // homeUrl=replyUserInfo.select(".authi").select("a").get(0).attr("href");
                                        String homeUrl = "";
                                        // String
                                        // replyUserId=getUrlParams(homeUrl).get("uid").toString();
                                        // System.out.println("头像内容："+replyUserInfo.select(".avatar"));
                                        String replyUserPhoto = "http://ucserver.higo.cn/avatar.php?uid=" + replyUserId + "&size=middle";
                                        //System.out.println("回复者头像："+replyUserPhoto);
                                        String replyContent = replyEntities.get(i).select(".plc").select(".pct").get(0).text();
                                        Elements replyImgEles=replyEntities.get(i).select(".plc").select(".pct").get(0).select("img[zoomfile]");
                                        JSONArray img_urls=new JSONArray();
                                        for(int imgIndex=0;imgIndex<replyImgEles.size();imgIndex++){
                                            Element imgEle=replyImgEles.get(imgIndex);
                                            String img_url=imgEle.absUrl("zoomfile");
                                            img_urls.add(img_url);
                                        }

                                        //System.out.println("回复内容："+replyContent);
                                        String replyTime = "";
                                        if (null != replyEntities.get(i).select(".authi").select("em").get(0).select("span") && replyEntities.get(i).select(".authi").select("em").get(0).select("span").size() > 0) {
                                            replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".authi").select("em").get(0).select("span").attr("title")) + ":00");
                                        } else {
                                            replyTime = DateUtils.strToFormatStr(getTime(replyEntities.get(i).select(".authi").select("em").text()) + ":00");
                                        }
                                        //System.out.println("回复时间："+replyTime);
                                        String replyUrl = url + "#" + replyEntities.get(i).id();
                                        // System.out.println("回复url："+replyUrl);

                                        reply.put("publish_user", (replyUser));
                                        reply.put("publish_user_id", replyUserId);
                                        reply.put("publish_user_photo", replyUserPhoto);
                                        reply.put("content", (replyContent));
                                        reply.put("publish_time", replyTime);
                                        reply.put("url", replyUrl);
                                        reply.put("parent_id", parent_id);
                                        reply.put("ancestor_id", parent_id);
                                        reply.put("img_urls",img_urls);
                                        arrayPage.add(reply);

                                    } catch (Exception e) {
                                        logger.error("因为禁言无法获取的情况",e);
                                        continue;// 因为禁言无法获取的情况
                                    }
                                }
                                flag = false;
                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                            } else {

                            }

                        } catch (Exception e1) {
                            //System.out.println("异常信息：" + e1.getMessage());

                            logger.error(e1.getMessage(),e1);
                        } finally {
                            if (flag == false) {
                                array.addAll(arrayPage);
                            }
                            nowTime = System.currentTimeMillis();
                        }
                    }

                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    continue;
                }
            }
            return array;
        }

        public ReplyCrawler(XiQinHuiGuanDownLoader downLoader, ArrayList<Integer> pages, String url, String parent_id, String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url = url;
            this.parent_id = parent_id;
            this.speeches_id = speeches_id;
        }


    }
}
